In [1]:
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn import linear_model
from sklearn.linear_model import Lasso
from sklearn.linear_model import Ridge
from sklearn.linear_model import LinearRegression
from sklearn.model_selection import GridSearchCV
import warnings
warnings.filterwarnings('ignore')
In [2]:
pd.set_option('display.max_column', 100)
In [3]:
df = pd.read_csv('train.csv', index_col=None)
df.head()
Out[3]:
In [4]:
df.info()
In [5]:
round(df.isnull().sum()/len(df.index), 2)>0.4
Out[5]:
In [ ]:
1. Data Cleaning & EDA¶
Handling Missing Values¶
There are 4 primary columns having more than 50% missing values in them. Also, looking at the column decription they can be removed because there are other variables present in the data serving same business descriptiom. For e.g.
- PoolArea defines the size of pool present, thus PoolQC can be remvoed as of now because of alot missing values. Although it can be an important factor in determining hourse price. Similary for FireplaceQu, it can be an important factor but not in all cases since there is already a varibale Fireplace defining presence/number of Fireplace at a hourse.
- Features like Fence, MisFeatures, LotFrontage and Alley seems important but have lot of missing values.
In [6]:
drop50 = ['Alley', 'PoolQC', 'Fence', 'MiscFeature', 'FireplaceQu']
# Dropping columns with high missing values
df = df.drop(drop50, axis=1)
In [7]:
# sum it up to check how many rows have all missing values
df.isnull().all(axis=1).sum()
Out[7]:
In [8]:
# Number of rows having missing values greater 5, 7, 10
print(len(df[df.isnull().sum(axis=1) > 5].index)/len(df))
print(len(df[df.isnull().sum(axis=1) > 7].index)/len(df))
print(len(df[df.isnull().sum(axis=1) > 10].index)/len(df))
In [9]:
df = df.drop(['LotFrontage'], axis=1)
In [10]:
# Dropping all rows in the dara-set having more than 5 missing values. Since the
df = df[df.isnull().sum(axis=1) <= 5]
print(df.shape)
df.info()
In [11]:
# Number of rows having missing values greater 2, 3, 4
print(len(df[df.isnull().sum(axis=1) > 2].index)/len(df))
print(len(df[df.isnull().sum(axis=1) > 3].index)/len(df))
print(len(df[df.isnull().sum(axis=1) > 4].index)/len(df))
In [12]:
len(df[df.isnull().sum(axis=1) > 2].index)
Out[12]:
Categorical and Numerical Variable Analysis of Missing Values¶
In [13]:
numerical = [var for var in df.columns if df[var].dtype!='O']
categorical = [var for var in df.columns if df[var].dtype=='O']
In [14]:
df[numerical].isnull().sum()
Out[14]:
In [15]:
df[categorical].isnull().sum()
Out[15]:
In [16]:
pd.set_option('display.max_rows', 150)
In [17]:
df[df.isnull().sum(axis=1)>2][['GarageType', 'GarageFinish', 'GarageQual', 'GarageCond']]
Out[17]:
In [18]:
df[df.isnull().sum(axis=1)>2][['BsmtQual', 'BsmtCond', 'BsmtExposure', 'BsmtFinType1', 'BsmtFinType2']]
Out[18]:
In [ ]:
Missing Values for Garage and Basement Details¶
- There are 74 houses for which Garage details are missing
- There are 30 hourses for Basement details are missing
This means that either the houses don't have garage and basement or this error is due to data recording. It's better to remove these values since no garage has already been taken care of in categories which means it's an error in data recording.
In [19]:
df = df[df.isnull().sum(axis=1) <= 2]
print(df.shape)
df.info()
Missing Value Imputation¶
In [20]:
# Imputing mean valus in numerical variable - MasVnrArea
df.loc[np.isnan(df['MasVnrArea']), 'MasVnrArea'] = df['MasVnrArea'].mean()
In [21]:
# Imputing Categorical Variables with most frequent values
df['MasVnrType'] = df['MasVnrType'].astype('category')
print(df['MasVnrType'].value_counts())
df['BsmtExposure'] = df['BsmtExposure'].astype('category')
print(df['BsmtExposure'].value_counts())
df['Electrical'] = df['Electrical'].astype('category')
print(df['Electrical'].value_counts())
df['BsmtFinType2'] = df['BsmtFinType2'].astype('category')
print(df['BsmtFinType2'].value_counts())
In [22]:
df['MasVnrType'] = df['MasVnrType'].fillna('None')
df['BsmtExposure'] = df['BsmtExposure'].fillna('No')
df['Electrical'] = df['Electrical'].fillna('SBrkr')
df['BsmtFinType2'] = df['BsmtFinType2'].fillna('Unf')
In [23]:
df.isnull().sum()>0
Out[23]:
In [24]:
df.info()
In [25]:
numerical = [var for var in df.columns if df[var].dtype!='O']
categorical = [var for var in df.columns if df[var].dtype=='O']
print(len(numerical))
print(len(categorical))
In [26]:
# Pairwise scatter plot for numerical variables
numeric1 = numerical[:20]
numeric2 = numerical[20:]
plt.figure(figsize=(20, 10))
sns.pairplot(df[numeric1])
plt.show()
In [27]:
plt.figure(figsize=(20, 10))
sns.pairplot(df[numeric2])
plt.show()
In [28]:
num_cat = [i for i in numerical if len(df[i].unique())<=5]
num_cat
Out[28]:
In [29]:
plt.figure(figsize = (22, 20))
sns.heatmap(df.corr(), cmap="YlGnBu", annot = True)
plt.show()
In [30]:
df.describe()
Out[30]:
In [31]:
sns.distplot(df['SalePrice'])
plt.show()
In [32]:
from datetime import datetime
year = datetime.now().year
Feature Engineering¶
In [33]:
df['house_age'] = year - df['YearBuilt']
df['house_age']
Out[33]:
In [34]:
sns.distplot(df['house_age'])
plt.show()
In [35]:
df = df.drop(['YearBuilt', 'YearRemodAdd'], axis=1)
In [36]:
numerical.remove('YearBuilt')
numerical.remove('YearRemodAdd')
In [37]:
df.shape
Out[37]:
In [ ]:
In [38]:
num_cat.append('BsmtFinType2')
categorical.extend(num_cat)
numerical = [i for i in numerical if i not in num_cat]
print(len(categorical))
print(len(numerical))
In [39]:
numerical.remove('Id')
In [40]:
# Scaling numerical variables using MinMaxScaler
from sklearn.preprocessing import MinMaxScaler
mms = MinMaxScaler()
df[numerical] = mms.fit_transform(df[numerical])
df.head()
Out[40]:
In [41]:
_list = ['BsmtFullBath', 'BsmtHalfBath', 'FullBath', 'HalfBath', 'KitchenAbvGr', 'Fireplaces', 'GarageCars', 'YrSold']
categorical = [i for i in categorical if i not in _list]
In [42]:
dfcategorical = df[categorical]
dfcategorical.head()
Out[42]:
In [43]:
dummies = pd.get_dummies(dfcategorical, drop_first=True)
dummies.head()
Out[43]:
In [44]:
# drop categorical variables
df = df.drop(categorical, axis=1)
# concat dummy variables with X
df = pd.concat([df, dummies], axis=1)
In [45]:
df.head()
Out[45]:
In [46]:
X = df.drop(['Id', 'SalePrice'], axis=1)
y = df['SalePrice']
In [47]:
# split into train and test
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=0.8, test_size = 0.2, random_state=100)
In [ ]:
In [48]:
# list of alphas to tune
params = {'alpha': [0.00005, 0.0001, 0.0002, 0.0003, 0.0005, 0.0007, 0.0009, 0.001, 0.005, 0.007, 0.01, 0.05, 0.07,
0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]}
ridge = Ridge()
# cross validation
folds = 5
model_cv = GridSearchCV(estimator = ridge, param_grid = params, scoring= 'r2', cv = folds,
return_train_score=True, verbose = 1)
model_cv.fit(X_train, y_train)
Out[48]:
In [49]:
cv_results = pd.DataFrame(model_cv.cv_results_)
cv_results = cv_results[cv_results['mean_train_score']>=0.85]
cv_results.head()
Out[49]:
In [50]:
# plotting mean test and train scoes with alpha
cv_results['param_alpha'] = cv_results['param_alpha'].astype('int32')
# plotting
plt.plot(cv_results['param_alpha'], cv_results['mean_train_score'])
plt.plot(cv_results['param_alpha'], cv_results['mean_test_score'])
plt.xlabel('alpha')
plt.ylabel('R^2 Error')
plt.title("R^2 Vs alpha")
plt.legend(['train score', 'test score'], loc='upper left')
plt.show()
In [51]:
alpha = 0.0001
ridge = Ridge(alpha=alpha)
ridge.fit(X_train, y_train)
ridge.coef_
Out[51]:
In [ ]:
Lasso¶
In [52]:
lasso = Lasso()
# cross validation
model_cv = GridSearchCV(estimator = lasso, param_grid = params, scoring= 'r2', cv = folds,
return_train_score=True, verbose = 1)
model_cv.fit(X_train, y_train)
Out[52]:
In [53]:
cv_results = pd.DataFrame(model_cv.cv_results_)
cv_results.head()
Out[53]:
In [54]:
# plotting mean test and train scoes with alpha
cv_results['param_alpha'] = cv_results['param_alpha'].astype('float32')
# plotting
plt.plot(cv_results['param_alpha'], cv_results['mean_train_score'])
plt.plot(cv_results['param_alpha'], cv_results['mean_test_score'])
plt.xscale('log')
plt.xlabel('alpha')
plt.ylabel('R^2 Error')
plt.title("R^2 Vs alpha")
plt.legend(['train score', 'test score'], loc='upper left')
plt.show()
In [55]:
alpha = 0.0001
lasso = Lasso(alpha=alpha)
lasso.fit(X_train, y_train)
Out[55]:
In [56]:
lasso.coef_
Out[56]:
In [ ]:
In [57]:
from sklearn.metrics import r2_score
In [58]:
y_pred = ridge.predict(X_test)
print('Ridge Test R2 Score', r2_score(y_test, y_pred))
In [59]:
y_pred = lasso.predict(X_test)
print('Lasso Test R2 Score', r2_score(y_test, y_pred))
In [ ]:
In [60]:
from sklearn.feature_selection import RFE
rfe = RFE(ridge, 10)
rfe = rfe.fit(X_train, y_train)
col1 = list(X_train.columns[rfe.support_])
col1
Out[60]:
In [61]:
rfe = RFE(lasso, 10)
rfe = rfe.fit(X_train, y_train)
col2 = list(X_train.columns[rfe.support_])
col2
Out[61]:
In [ ]:
Analysis & Model Report¶
In [62]:
finalcol = [i for i in col1 if i in col2]
finalcol
Out[62]:
In [63]:
col1.append('SalePrice')
col2.append('SalePrice')
In [64]:
plt.figure(figsize = (12, 8))
sns.heatmap(df[col1].corr(), cmap="YlGnBu", annot = True)
plt.show()
In [65]:
plt.figure(figsize = (12, 8))
sns.heatmap(df[col2].corr(), cmap="YlGnBu", annot = True)
plt.show()
In [ ]:
Top Variables that are significant in predicting the price of a house¶
- OverallQual - Rating of overall material and finish of the house
- BsmtFinSF1 - Type 1 finished square feet
- TotalBsmtSF - Total square feet of basement area
- Condition2 - Proximity to various conditions (PosN - Near positive off-site feature--park, greenbelt, etc.)
- GrLivArea - Above grade (ground) living area square feet
- GarageArea - Size of garage in square feet
- 1stFlrSF - First Floor square feet
- LotArea - Lot size in square feet
- MasVnrArea - Masonry veneer area in square feet
Optimal Value of lambda:¶
- Ridge - 0.0001
- Lasso - 0.0001
In [66]:
finalcol.append('GrLivArea')
finalcol.append('GarageArea')
finalcol.append('1stFlrSF')
finalcol.append('LotArea')
finalcol.append('MasVnrArea')
In [67]:
finalcol
Out[67]:
In [68]:
xtrain_new = X_train[finalcol]
xtest_new = X_test[finalcol]
In [69]:
lasso = Lasso()
# cross validation
model_cv = GridSearchCV(estimator = lasso, param_grid = params, scoring= 'r2', cv = folds,
return_train_score=True, verbose = 1)
model_cv.fit(xtrain_new, y_train)
Out[69]:
In [70]:
# plotting mean test and train scoes with alpha
cv_results['param_alpha'] = cv_results['param_alpha'].astype('float32')
# plotting
plt.plot(cv_results['param_alpha'], cv_results['mean_train_score'])
plt.plot(cv_results['param_alpha'], cv_results['mean_test_score'])
plt.xscale('log')
plt.xlabel('alpha')
plt.ylabel('R^2')
plt.title("R^2 Vs alpha")
plt.legend(['train score', 'test score'], loc='upper left')
plt.show()
In [ ]: